Snapshot analysis of moea-benchmark data
Reproducing the plots and tables from the ECJ paper.
This notebook has been rendered as an HTML page for your navigation. Yet, the notebook is also available for cloning, or to be executed online using Binder or Colab.
Below, you will find the figures and tables from the ECJ paper that consider only outputs at given values of maximum function evaluations ($\textit{FE}_\textit{max}$), which we dub snapshot analysis.
In the paper, we mostly focused on $\textit{FE}_\textit{max}=10000$ results. In this notebook, results are first presented as in the paper, and then provided for more experimental scenarios, when possible.
Setup
The data for snapshot analysis is provided in the original moea-benchmark repository, and can be readily using the pandas data science library for Python.
Besides pandas, we will also use the Plotly interactive data visualization library.
Finally, to improve plotting clarity, we define querys that remove outliers from the data, when necessary.
import re
import pandas as pd
import plotly.express as px
import plotly.graph_objects as go
df = pd.read_csv("https://github.com/leobezerra/moea-benchmark/raw/master/indicators.csv.gz")
df.head()
rpd_outliers = "indicator == 'rpd' and 0 <= value <= 0.4"
eps_outliers = "indicator == 'eps' and 0 <= value <= 4"
igd_outliers = "indicator == 'igd' and 0 <= value <= 10"
Since Figure 1 is a comparison between tuned and default settings of the MOEAs, we exclude MOGA from this analysis, as no default settings are available for this algorithm.
In addition, we remove outliers from this analysis to improve plotting clarity, as previously discussed.
df_no_moga = df.query("algo != 'moga'")
df_no_moga_nor_outliers = df_no_moga.query(f"{rpd_outliers} or {eps_outliers} or {igd_outliers}")
df_tuned_default = df_no_moga_nor_outliers.pivot_table(
index=["FE", "algo", "indicator", "nobj", "problem", "nvar"],
columns=["setup"],
values=["value"]
)
df_tuned_default = df_tuned_default.droplevel(0, axis=1).reset_index()
df_tuned_default
# Auxiliary procedure to generate diagonal lines
def add_line(fig, xmax, row):
fig.add_scatter(
x=[0, xmax],
y=[0, xmax],
mode="lines",
line=go.scatter.Line(color="red"),
row=row,
col=1,
showlegend=False,
)
fig = px.scatter(
df_tuned_default,
x="tuned",
y="default",
facet_row="indicator",
category_orders={"indicator": ["rpd", "eps", "igd"]},
height=800,
width=400,
)
for k in fig.layout:
if re.search('xaxis[1-9]+', k):
fig.layout[k].update(matches=None)
for k in fig.layout:
if re.search('yaxis[1-9]+', k):
fig.layout[k].update(matches=None)
add_line(fig, 0.4, 3)
add_line(fig, 4, 2)
add_line(fig, 10, 1)
Table 6 computes Pearson’s correlation coefficient between MOEA rankings for different values of $\textit{FE}_\textit{max}$.
We remark that the rankings for tuned MOEAs are computed including MOGA.
def rank_sum(df, columns=["algo"]):
df_wide = df.pivot_table(
index=["indicator", "problem", "nvar", "seed"],
columns=columns,
values=["value"]
)
return df_wide.rank(axis=1).groupby("indicator").sum()
df_rs = df.groupby(["setup","FE", "nobj"]).apply(rank_sum).droplevel(0, axis=1)
long_rs_tuned = df_rs.query("setup == 'tuned'").stack().reset_index(name="value")
df_rs_tuned = long_rs_tuned.pivot_table(
index=["indicator", "nobj", "algo"],
columns=["FE"],
values=["value"]
).droplevel(0, axis=1)
long_rs_default = df_rs.query("setup == 'default'").stack().reset_index(name="value")
df_rs_default = long_rs_default.pivot_table(
index=["indicator", "nobj", "algo"],
columns=["FE"],
values=["value"]
).droplevel(0, axis=1)
Figure 5 compares IBEA and SMS directly according to the $\textit{HV}_\textit{rd}$ and $\textit{IGD}$ indicators, on a specific experimental scenario.
algo_fig5 = ["ibea", "sms"]
ind_fig5 = ["rpd", "igd"]
scenario_fig5 = "problem == 'WFG8' and nobj == 2 and nvar == 30 and FE == 10000"
df_fig5 = df.query(f"algo in {algo_fig5} and indicator in {ind_fig5} and {scenario_fig5} and setup == 'tuned'")
fig5 = px.box(
df_fig5,
y="algo",
x="value",
color="algo",
facet_col="indicator",
height=300,
width=600,
)
Alternatively, we also provide code to produce the full set of boxplots from the data produced in the paper.
Note that the code provided has been adjusted to improve clarity, but can be configured in any way desired.
In addition, a few resources from Plotly can be useful for navigation:
- selecting a subset of the MOEAs, by clicking on their names in the legend
- zooming into a given range of a given plot, by selecting an area of the plot
df_tuned_no_outliers_10k = df.query(f"setup == 'tuned' and FE == 10000 and {rpd_outliers} or {eps_outliers} or {igd_outliers}")
fig5_full = px.box(
df_tuned_no_outliers,
x="nvar",
y="value",
color="algo",
facet_col="nobj",
facet_row="indicator",
animation_frame="problem",
height=1000,
category_orders={"indicator": ["rpd", "eps", "igd"]}
)
ymax = [10, 4, 0.4]
for k in fig5_full.layout:
if re.search('yaxis[1-9]*', k):
matches = re.findall(r'(\d+)', k)
idx = int(matches[0]) if len(matches) else 1
ymax_idx = (idx-1) // 4
fig5_full.layout[k].update(matches=None, range=(0,ymax[ymax_idx]))
Table 7 computes Pearson’s correlation coefficient between MOEA rankings for different performance metrics.
Like done for Table 6, tuned MOEA rankings are computed including MOGA.
for FE in [2500, 10000, 40000]:
for nobj in [2,3,5,10]:
for indicator in ["rpd", "eps", "igd"]:
idx = ("tuned", FE, nobj, indicator)
rs_diff = (df_rs.loc[idx] - df_rs.loc[idx].min())
display(rs_diff.sort_values().to_frame().T)
algo_tab9 = ["nsga", "nsga3"]
nobj_tab9 = [2,5]
df_nsga_tab9 = df.query(f"algo in {algo_tab9} and indicator == 'rpd' and nobj in {nobj_tab9} and FE == 10000")
rs_nsga_tab9 = df_nsga_tab9.groupby("nobj").apply(rank_sum, columns=["setup","algo"]).droplevel(0, axis=1)
for nobj in [2,5]:
idx = (nobj, "rpd")
rs_diff_tab9 = (rs_nsga_tab9.loc[idx] - rs_nsga_tab9.loc[idx].min())
display(rs_diff_tab9.sort_values().to_frame().T)
df_rpd_10k = df.query(f"indicator == 'rpd' and FE == 10000 and setup == 'tuned' and {rpd_outliers}")
df_rpd_10k["nobj"] = df_rpd_10k["nobj"].astype("category")
px.box(
df_rpd_10k,
x="nobj",
color="nobj",
y="value",
facet_col="problem",
facet_col_wrap=7,
range_y=(0,0.4),
points=False,
)
problem_fig8 = ["WFG1", "WFG4"]
nvar_fig8 = [40, 41]
setup_fig8 = "FE == 40000 and indicator == 'igd' and setup == 'tuned'"
df_fig8 = df.query(f"problem in {problem_fig8} and nvar in {nvar_fig8} and {setup_fig8} and {igd_outliers}")
fig8 = px.box(
df_fig8,
x="value",
y="algo",
color="algo",
facet_col="nobj",
facet_row="problem",
category_orders={"algo": ["cma", "hype", "ibea", "moead", "moga", "nsga", "nsga3", "sms", "spea"][::-1]},
)
fig8.show()